{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), November 20, 2020**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split-apply-combine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grouping a data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>8 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>2</td><td>2</td><td>0.946112</td></tr><tr><th>3</th><td>3</td><td>1</td><td>0.759872</td></tr><tr><th>4</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>5</th><td>1</td><td>1</td><td>0.677472</td></tr><tr><th>6</th><td>2</td><td>2</td><td>0.802295</td></tr><tr><th>7</th><td>3</td><td>1</td><td>0.906859</td></tr><tr><th>8</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 2 & 2 & 0.946112 \\\\\n",
       "\t3 & 3 & 1 & 0.759872 \\\\\n",
       "\t4 & 4 & 2 & 0.292012 \\\\\n",
       "\t5 & 1 & 1 & 0.677472 \\\\\n",
       "\t6 & 2 & 2 & 0.802295 \\\\\n",
       "\t7 & 3 & 1 & 0.906859 \\\\\n",
       "\t8 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m8×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     2      2  0.946112\n",
       "   3 │     3      1  0.759872\n",
       "   4 │     4      2  0.292012\n",
       "   5 │     1      1  0.677472\n",
       "   6 │     2      2  0.802295\n",
       "   7 │     3      1  0.906859\n",
       "   8 │     4      2  0.474816"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(id=[1,2,3,4,1,2,3,4], id2=[1,2,1,2,1,2,1,2], v=rand(8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on key: id</b></p><p><i>First Group (2 rows): id = 1</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (2 rows): id = 4</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>2</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on key: id\n",
       "\n",
       "First Group (2 rows): id = 1\n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (2 rows): id = 4\n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 4 & 2 & 0.292012 \\\\\n",
       "\t2 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on key: id\n",
       "First Group (2 rows): id = 1\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472\n",
       "⋮\n",
       "Last Group (2 rows): id = 4\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     4      2  0.292012\n",
       "   2 │     4      2  0.474816"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groupby(x, :id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 1 group based on key: </b></p><p><i>First Group (8 rows): </i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>2</td><td>2</td><td>0.946112</td></tr><tr><th>3</th><td>3</td><td>1</td><td>0.759872</td></tr><tr><th>4</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>5</th><td>1</td><td>1</td><td>0.677472</td></tr><tr><th>6</th><td>2</td><td>2</td><td>0.802295</td></tr><tr><th>7</th><td>3</td><td>1</td><td>0.906859</td></tr><tr><th>8</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 1 group based on key: \n",
       "\n",
       "First Group (8 rows): \n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 2 & 2 & 0.946112 \\\\\n",
       "\t3 & 3 & 1 & 0.759872 \\\\\n",
       "\t4 & 4 & 2 & 0.292012 \\\\\n",
       "\t5 & 1 & 1 & 0.677472 \\\\\n",
       "\t6 & 2 & 2 & 0.802295 \\\\\n",
       "\t7 & 3 & 1 & 0.906859 \\\\\n",
       "\t8 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 1 group based on key: \n",
       "First Group (8 rows): \n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     2      2  0.946112\n",
       "   3 │     3      1  0.759872\n",
       "   4 │     4      2  0.292012\n",
       "   5 │     1      1  0.677472\n",
       "   6 │     2      2  0.802295\n",
       "   7 │     3      1  0.906859\n",
       "   8 │     4      2  0.474816"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groupby(x, [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on keys: id, id2</b></p><p><i>First Group (2 rows): id = 1, id2 = 1</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (2 rows): id = 4, id2 = 2</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>2</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on keys: id, id2\n",
       "\n",
       "First Group (2 rows): id = 1, id2 = 1\n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (2 rows): id = 4, id2 = 2\n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 4 & 2 & 0.292012 \\\\\n",
       "\t2 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on keys: id, id2\n",
       "First Group (2 rows): id = 1, id2 = 1\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472\n",
       "⋮\n",
       "Last Group (2 rows): id = 4, id2 = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     4      2  0.292012\n",
       "   2 │     4      2  0.474816"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gx2 = groupby(x, [:id, :id2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>8 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>2</td><td>2</td><td>0.946112</td></tr><tr><th>3</th><td>3</td><td>1</td><td>0.759872</td></tr><tr><th>4</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>5</th><td>1</td><td>1</td><td>0.677472</td></tr><tr><th>6</th><td>2</td><td>2</td><td>0.802295</td></tr><tr><th>7</th><td>3</td><td>1</td><td>0.906859</td></tr><tr><th>8</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 2 & 2 & 0.946112 \\\\\n",
       "\t3 & 3 & 1 & 0.759872 \\\\\n",
       "\t4 & 4 & 2 & 0.292012 \\\\\n",
       "\t5 & 1 & 1 & 0.677472 \\\\\n",
       "\t6 & 2 & 2 & 0.802295 \\\\\n",
       "\t7 & 3 & 1 & 0.906859 \\\\\n",
       "\t8 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m8×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     2      2  0.946112\n",
       "   3 │     3      1  0.759872\n",
       "   4 │     4      2  0.292012\n",
       "   5 │     1      1  0.677472\n",
       "   6 │     2      2  0.802295\n",
       "   7 │     3      1  0.906859\n",
       "   8 │     4      2  0.474816"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parent(gx2) # get the parent DataFrame "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>8 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr><tr><th>3</th><td>2</td><td>2</td><td>0.946112</td></tr><tr><th>4</th><td>2</td><td>2</td><td>0.802295</td></tr><tr><th>5</th><td>3</td><td>1</td><td>0.759872</td></tr><tr><th>6</th><td>3</td><td>1</td><td>0.906859</td></tr><tr><th>7</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>8</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\t3 & 2 & 2 & 0.946112 \\\\\n",
       "\t4 & 2 & 2 & 0.802295 \\\\\n",
       "\t5 & 3 & 1 & 0.759872 \\\\\n",
       "\t6 & 3 & 1 & 0.906859 \\\\\n",
       "\t7 & 4 & 2 & 0.292012 \\\\\n",
       "\t8 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m8×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472\n",
       "   3 │     2      2  0.946112\n",
       "   4 │     2      2  0.802295\n",
       "   5 │     3      1  0.759872\n",
       "   6 │     3      1  0.906859\n",
       "   7 │     4      2  0.292012\n",
       "   8 │     4      2  0.474816"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vcat(gx2...) # back to the DataFrame, but in a different order of rows than the original"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>8 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr><tr><th>3</th><td>2</td><td>2</td><td>0.946112</td></tr><tr><th>4</th><td>2</td><td>2</td><td>0.802295</td></tr><tr><th>5</th><td>3</td><td>1</td><td>0.759872</td></tr><tr><th>6</th><td>3</td><td>1</td><td>0.906859</td></tr><tr><th>7</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>8</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\t3 & 2 & 2 & 0.946112 \\\\\n",
       "\t4 & 2 & 2 & 0.802295 \\\\\n",
       "\t5 & 3 & 1 & 0.759872 \\\\\n",
       "\t6 & 3 & 1 & 0.906859 \\\\\n",
       "\t7 & 4 & 2 & 0.292012 \\\\\n",
       "\t8 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m8×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472\n",
       "   3 │     2      2  0.946112\n",
       "   4 │     2      2  0.802295\n",
       "   5 │     3      1  0.759872\n",
       "   6 │     3      1  0.906859\n",
       "   7 │     4      2  0.292012\n",
       "   8 │     4      2  0.474816"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrame(gx2) # the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>v</th></tr><tr><th></th><th>Float64</th></tr></thead><tbody><p>8 rows × 1 columns</p><tr><th>1</th><td>0.998123</td></tr><tr><th>2</th><td>0.677472</td></tr><tr><th>3</th><td>0.946112</td></tr><tr><th>4</th><td>0.802295</td></tr><tr><th>5</th><td>0.759872</td></tr><tr><th>6</th><td>0.906859</td></tr><tr><th>7</th><td>0.292012</td></tr><tr><th>8</th><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|c}\n",
       "\t& v\\\\\n",
       "\t\\hline\n",
       "\t& Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 0.998123 \\\\\n",
       "\t2 & 0.677472 \\\\\n",
       "\t3 & 0.946112 \\\\\n",
       "\t4 & 0.802295 \\\\\n",
       "\t5 & 0.759872 \\\\\n",
       "\t6 & 0.906859 \\\\\n",
       "\t7 & 0.292012 \\\\\n",
       "\t8 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m8×1 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\n",
       "─────┼──────────\n",
       "   1 │ 0.998123\n",
       "   2 │ 0.677472\n",
       "   3 │ 0.946112\n",
       "   4 │ 0.802295\n",
       "   5 │ 0.759872\n",
       "   6 │ 0.906859\n",
       "   7 │ 0.292012\n",
       "   8 │ 0.474816"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrame(gx2, keepkeys=false) # drop grouping columns when creating a data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2-element Array{Symbol,1}:\n",
       " :id\n",
       " :id2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groupcols(gx2) # vector of names of grouping variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1-element Array{Symbol,1}:\n",
       " :v"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valuecols(gx2) # and non-grouping variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8-element Array{Union{Missing, Int64},1}:\n",
       " 1\n",
       " 2\n",
       " 3\n",
       " 4\n",
       " 1\n",
       " 2\n",
       " 3\n",
       " 4"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groupindices(gx2) # group indices in parent(gx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element DataFrames.GroupKeys{GroupedDataFrame{DataFrame}}:\n",
       " GroupKey: (id = 1, id2 = 1)\n",
       " GroupKey: (id = 2, id2 = 2)\n",
       " GroupKey: (id = 3, id2 = 1)\n",
       " GroupKey: (id = 4, id2 = 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kgx2 = keys(gx2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can index into a `GroupedDataFrame` like to a vector or to a dictionary.\n",
    "The second form acceps `GroupKey`, `NamedTuple` or a `Tuple`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on keys: id, id2</b></p><p><i>First Group (2 rows): id = 1, id2 = 1</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (2 rows): id = 4, id2 = 2</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>4</td><td>2</td><td>0.292012</td></tr><tr><th>2</th><td>4</td><td>2</td><td>0.474816</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on keys: id, id2\n",
       "\n",
       "First Group (2 rows): id = 1, id2 = 1\n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (2 rows): id = 4, id2 = 2\n",
       "\n",
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 4 & 2 & 0.292012 \\\\\n",
       "\t2 & 4 & 2 & 0.474816 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on keys: id, id2\n",
       "First Group (2 rows): id = 1, id2 = 1\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472\n",
       "⋮\n",
       "Last Group (2 rows): id = 4, id2 = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     4      2  0.292012\n",
       "   2 │     4      2  0.474816"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gx2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GroupKey: (id = 1, id2 = 1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k = keys(gx2)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(id = 1, id2 = 1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ntk = NamedTuple(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tk = Tuple(k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the operations below produce the same result and are fast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>2 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×3 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gx2[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>2 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×3 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gx2[k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>2 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×3 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gx2[ntk]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>2 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>0.998123</td></tr><tr><th>2</th><td>1</td><td>1</td><td>0.677472</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 0.998123 \\\\\n",
       "\t2 & 1 & 1 & 0.677472 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×3 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────────────\n",
       "   1 │     1      1  0.998123\n",
       "   2 │     1      1  0.677472"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gx2[tk]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "handling missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64?</th><th>Int64</th></tr></thead><tbody><p>5 rows × 2 columns</p><tr><th>1</th><td><em>missing</em></td><td>1</td></tr><tr><th>2</th><td>5</td><td>2</td></tr><tr><th>3</th><td>1</td><td>3</td></tr><tr><th>4</th><td>3</td><td>4</td></tr><tr><th>5</th><td><em>missing</em></td><td>5</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64? & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & \\emph{missing} & 1 \\\\\n",
       "\t2 & 5 & 2 \\\\\n",
       "\t3 & 1 & 3 \\\\\n",
       "\t4 & 3 & 4 \\\\\n",
       "\t5 & \\emph{missing} & 5 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m5×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id      \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64?  \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼────────────────\n",
       "   1 │\u001b[90m missing \u001b[0m     1\n",
       "   2 │       5      2\n",
       "   3 │       1      3\n",
       "   4 │       3      4\n",
       "   5 │\u001b[90m missing \u001b[0m     5"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(id = [missing, 5, 1, 3, missing], x = 1:5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on key: id</b></p><p><i>First Group (2 rows): id = missing</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64?</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td><em>missing</em></td><td>1</td></tr><tr><th>2</th><td><em>missing</em></td><td>5</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (1 row): id = 3</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64?</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>3</td><td>4</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on key: id\n",
       "\n",
       "First Group (2 rows): id = missing\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64? & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & \\emph{missing} & 1 \\\\\n",
       "\t2 & \\emph{missing} & 5 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (1 row): id = 3\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64? & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 3 & 4 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on key: id\n",
       "First Group (2 rows): id = missing\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id      \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64?  \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼────────────────\n",
       "   1 │\u001b[90m missing \u001b[0m     1\n",
       "   2 │\u001b[90m missing \u001b[0m     5\n",
       "⋮\n",
       "Last Group (1 row): id = 3\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id     \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64? \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │      3      4"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groupby(x, :id) # by default groups include mising values and are not sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 3 groups based on key: id</b></p><p><i>First Group (1 row): id = 1</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64?</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>3</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (1 row): id = 5</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64?</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>5</td><td>2</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 3 groups based on key: id\n",
       "\n",
       "First Group (1 row): id = 1\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64? & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 3 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (1 row): id = 5\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64? & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 5 & 2 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 3 groups based on key: id\n",
       "First Group (1 row): id = 1\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id     \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64? \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │      1      3\n",
       "⋮\n",
       "Last Group (1 row): id = 5\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id     \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64? \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │      5      2"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groupby(x, :id, sort=true, skipmissing=true) # but we can change it"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performing transformations by group using `combine`, `select`, `select!`, `transform`, and `transform!`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Statistics\n",
    "using Pipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ENV[\"LINES\"] = 15 # reduce the number of rows in the output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>v</th></tr><tr><th></th><th>Char</th><th>Float64</th></tr></thead><tbody><p>100 rows × 2 columns</p><tr><th>1</th><td>b</td><td>0.494644</td></tr><tr><th>2</th><td>a</td><td>0.720003</td></tr><tr><th>3</th><td>a</td><td>0.221093</td></tr><tr><th>4</th><td>a</td><td>0.27621</td></tr><tr><th>5</th><td>d</td><td>0.205227</td></tr><tr><th>6</th><td>c</td><td>0.624687</td></tr><tr><th>7</th><td>c</td><td>0.397678</td></tr><tr><th>8</th><td>b</td><td>0.464126</td></tr><tr><th>9</th><td>a</td><td>0.993339</td></tr><tr><th>10</th><td>d</td><td>0.541395</td></tr><tr><th>11</th><td>c</td><td>0.903439</td></tr><tr><th>12</th><td>d</td><td>0.970156</td></tr><tr><th>13</th><td>d</td><td>0.653396</td></tr><tr><th>14</th><td>b</td><td>0.168943</td></tr><tr><th>15</th><td>a</td><td>0.11324</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & v\\\\\n",
       "\t\\hline\n",
       "\t& Char & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 0.494644 \\\\\n",
       "\t2 & a & 0.720003 \\\\\n",
       "\t3 & a & 0.221093 \\\\\n",
       "\t4 & a & 0.27621 \\\\\n",
       "\t5 & d & 0.205227 \\\\\n",
       "\t6 & c & 0.624687 \\\\\n",
       "\t7 & c & 0.397678 \\\\\n",
       "\t8 & b & 0.464126 \\\\\n",
       "\t9 & a & 0.993339 \\\\\n",
       "\t10 & d & 0.541395 \\\\\n",
       "\t11 & c & 0.903439 \\\\\n",
       "\t12 & d & 0.970156 \\\\\n",
       "\t13 & d & 0.653396 \\\\\n",
       "\t14 & b & 0.168943 \\\\\n",
       "\t15 & a & 0.11324 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m100×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m v        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────\n",
       "   1 │ b     0.494644\n",
       "   2 │ a     0.720003\n",
       "   3 │ a     0.221093\n",
       "  ⋮  │  ⋮       ⋮\n",
       "  98 │ c     0.105981\n",
       "  99 │ d     0.749458\n",
       " 100 │ d     0.16598\n",
       "\u001b[31m       94 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(id=rand('a':'d', 100), v=rand(100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>v_mean</th></tr><tr><th></th><th>Char</th><th>Float64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>b</td><td>0.489617</td></tr><tr><th>2</th><td>a</td><td>0.540168</td></tr><tr><th>3</th><td>d</td><td>0.529731</td></tr><tr><th>4</th><td>c</td><td>0.50889</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & v\\_mean\\\\\n",
       "\t\\hline\n",
       "\t& Char & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 0.489617 \\\\\n",
       "\t2 & a & 0.540168 \\\\\n",
       "\t3 & d & 0.529731 \\\\\n",
       "\t4 & c & 0.50889 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m v_mean   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────\n",
       "   1 │ b     0.489617\n",
       "   2 │ a     0.540168\n",
       "   3 │ d     0.529731\n",
       "   4 │ c     0.50889"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# apply a function to each group of a data frame\n",
    "# combine keeps as many rows as are returned from the function\n",
    "@pipe x |> groupby(_, :id) |> combine(_, :v=>mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Base.OneTo(100)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.id2 = axes(x, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>v</th><th>id2</th><th>v_mean</th></tr><tr><th></th><th>Char</th><th>Float64</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>100 rows × 4 columns</p><tr><th>1</th><td>b</td><td>0.494644</td><td>1</td><td>0.489617</td></tr><tr><th>2</th><td>a</td><td>0.720003</td><td>2</td><td>0.540168</td></tr><tr><th>3</th><td>a</td><td>0.221093</td><td>3</td><td>0.540168</td></tr><tr><th>4</th><td>a</td><td>0.27621</td><td>4</td><td>0.540168</td></tr><tr><th>5</th><td>d</td><td>0.205227</td><td>5</td><td>0.529731</td></tr><tr><th>6</th><td>c</td><td>0.624687</td><td>6</td><td>0.50889</td></tr><tr><th>7</th><td>c</td><td>0.397678</td><td>7</td><td>0.50889</td></tr><tr><th>8</th><td>b</td><td>0.464126</td><td>8</td><td>0.489617</td></tr><tr><th>9</th><td>a</td><td>0.993339</td><td>9</td><td>0.540168</td></tr><tr><th>10</th><td>d</td><td>0.541395</td><td>10</td><td>0.529731</td></tr><tr><th>11</th><td>c</td><td>0.903439</td><td>11</td><td>0.50889</td></tr><tr><th>12</th><td>d</td><td>0.970156</td><td>12</td><td>0.529731</td></tr><tr><th>13</th><td>d</td><td>0.653396</td><td>13</td><td>0.529731</td></tr><tr><th>14</th><td>b</td><td>0.168943</td><td>14</td><td>0.489617</td></tr><tr><th>15</th><td>a</td><td>0.11324</td><td>15</td><td>0.540168</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccc}\n",
       "\t& id & v & id2 & v\\_mean\\\\\n",
       "\t\\hline\n",
       "\t& Char & Float64 & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 0.494644 & 1 & 0.489617 \\\\\n",
       "\t2 & a & 0.720003 & 2 & 0.540168 \\\\\n",
       "\t3 & a & 0.221093 & 3 & 0.540168 \\\\\n",
       "\t4 & a & 0.27621 & 4 & 0.540168 \\\\\n",
       "\t5 & d & 0.205227 & 5 & 0.529731 \\\\\n",
       "\t6 & c & 0.624687 & 6 & 0.50889 \\\\\n",
       "\t7 & c & 0.397678 & 7 & 0.50889 \\\\\n",
       "\t8 & b & 0.464126 & 8 & 0.489617 \\\\\n",
       "\t9 & a & 0.993339 & 9 & 0.540168 \\\\\n",
       "\t10 & d & 0.541395 & 10 & 0.529731 \\\\\n",
       "\t11 & c & 0.903439 & 11 & 0.50889 \\\\\n",
       "\t12 & d & 0.970156 & 12 & 0.529731 \\\\\n",
       "\t13 & d & 0.653396 & 13 & 0.529731 \\\\\n",
       "\t14 & b & 0.168943 & 14 & 0.489617 \\\\\n",
       "\t15 & a & 0.11324 & 15 & 0.540168 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m100×4 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m v        \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v_mean   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼─────────────────────────────────\n",
       "   1 │ b     0.494644      1  0.489617\n",
       "   2 │ a     0.720003      2  0.540168\n",
       "   3 │ a     0.221093      3  0.540168\n",
       "  ⋮  │  ⋮       ⋮        ⋮       ⋮\n",
       "  98 │ c     0.105981     98  0.50889\n",
       "  99 │ d     0.749458     99  0.529731\n",
       " 100 │ d     0.16598     100  0.529731\n",
       "\u001b[31m                        94 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select and transform keep as many rows as are in the source data frame and in correct order\n",
    "# additionally transform keeps all columns from the source\n",
    "@pipe x |> groupby(_, :id) |> transform(_, :v=>mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>id2</th><th>v_mean</th></tr><tr><th></th><th>Char</th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>100 rows × 3 columns</p><tr><th>1</th><td>b</td><td>1</td><td>0.489617</td></tr><tr><th>2</th><td>b</td><td>8</td><td>0.489617</td></tr><tr><th>3</th><td>b</td><td>14</td><td>0.489617</td></tr><tr><th>4</th><td>b</td><td>16</td><td>0.489617</td></tr><tr><th>5</th><td>b</td><td>19</td><td>0.489617</td></tr><tr><th>6</th><td>b</td><td>29</td><td>0.489617</td></tr><tr><th>7</th><td>b</td><td>39</td><td>0.489617</td></tr><tr><th>8</th><td>b</td><td>41</td><td>0.489617</td></tr><tr><th>9</th><td>b</td><td>42</td><td>0.489617</td></tr><tr><th>10</th><td>b</td><td>44</td><td>0.489617</td></tr><tr><th>11</th><td>b</td><td>48</td><td>0.489617</td></tr><tr><th>12</th><td>b</td><td>50</td><td>0.489617</td></tr><tr><th>13</th><td>b</td><td>54</td><td>0.489617</td></tr><tr><th>14</th><td>b</td><td>55</td><td>0.489617</td></tr><tr><th>15</th><td>b</td><td>57</td><td>0.489617</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & id2 & v\\_mean\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 1 & 0.489617 \\\\\n",
       "\t2 & b & 8 & 0.489617 \\\\\n",
       "\t3 & b & 14 & 0.489617 \\\\\n",
       "\t4 & b & 16 & 0.489617 \\\\\n",
       "\t5 & b & 19 & 0.489617 \\\\\n",
       "\t6 & b & 29 & 0.489617 \\\\\n",
       "\t7 & b & 39 & 0.489617 \\\\\n",
       "\t8 & b & 41 & 0.489617 \\\\\n",
       "\t9 & b & 42 & 0.489617 \\\\\n",
       "\t10 & b & 44 & 0.489617 \\\\\n",
       "\t11 & b & 48 & 0.489617 \\\\\n",
       "\t12 & b & 50 & 0.489617 \\\\\n",
       "\t13 & b & 54 & 0.489617 \\\\\n",
       "\t14 & b & 55 & 0.489617 \\\\\n",
       "\t15 & b & 57 & 0.489617 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m100×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m id2   \u001b[0m\u001b[1m v_mean   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼───────────────────────\n",
       "   1 │ b         1  0.489617\n",
       "   2 │ b         8  0.489617\n",
       "   3 │ b        14  0.489617\n",
       "  ⋮  │  ⋮      ⋮       ⋮\n",
       "  98 │ c        89  0.50889\n",
       "  99 │ c        93  0.50889\n",
       " 100 │ c        98  0.50889\n",
       "\u001b[31m              94 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# note that combine reorders rows by group of GroupedDataFrame\n",
    "@pipe x |> groupby(_, :id) |> combine(_, :id2, :v=>mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>res</th></tr><tr><th></th><th>Char</th><th>Float64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>b</td><td>0.489617</td></tr><tr><th>2</th><td>a</td><td>0.540168</td></tr><tr><th>3</th><td>d</td><td>0.529731</td></tr><tr><th>4</th><td>c</td><td>0.50889</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & res\\\\\n",
       "\t\\hline\n",
       "\t& Char & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 0.489617 \\\\\n",
       "\t2 & a & 0.540168 \\\\\n",
       "\t3 & d & 0.529731 \\\\\n",
       "\t4 & c & 0.50889 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m res      \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Float64  \u001b[0m\n",
       "─────┼────────────────\n",
       "   1 │ b     0.489617\n",
       "   2 │ a     0.540168\n",
       "   3 │ d     0.529731\n",
       "   4 │ c     0.50889"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we give a custom name for the result column\n",
    "@pipe x |> groupby(_, :id) |> combine(_, :v=>mean=>:res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>res1</th><th>res2</th><th>n</th></tr><tr><th></th><th>Char</th><th>Float64</th><th>Float64</th><th>Int64</th></tr></thead><tbody><p>4 rows × 4 columns</p><tr><th>1</th><td>b</td><td>0.489617</td><td>11.2612</td><td>23</td></tr><tr><th>2</th><td>a</td><td>0.540168</td><td>12.964</td><td>24</td></tr><tr><th>3</th><td>d</td><td>0.529731</td><td>11.6541</td><td>22</td></tr><tr><th>4</th><td>c</td><td>0.50889</td><td>15.7756</td><td>31</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccc}\n",
       "\t& id & res1 & res2 & n\\\\\n",
       "\t\\hline\n",
       "\t& Char & Float64 & Float64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 0.489617 & 11.2612 & 23 \\\\\n",
       "\t2 & a & 0.540168 & 12.964 & 24 \\\\\n",
       "\t3 & d & 0.529731 & 11.6541 & 22 \\\\\n",
       "\t4 & c & 0.50889 & 15.7756 & 31 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×4 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m res1     \u001b[0m\u001b[1m res2    \u001b[0m\u001b[1m n     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼────────────────────────────────\n",
       "   1 │ b     0.489617  11.2612     23\n",
       "   2 │ a     0.540168  12.964      24\n",
       "   3 │ d     0.529731  11.6541     22\n",
       "   4 │ c     0.50889   15.7756     31"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you can have multiple operations\n",
    "@pipe x |> groupby(_, :id) |> combine(_, :v=>mean=>:res1, :v=>sum=>:res2, nrow=>:n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Additional notes:\n",
    "* `select!` and `transform!` perform operations in-place\n",
    "* The general syntax for transformation is `source_columns => function => target_column`\n",
    "* if you pass multiple columns to a function they are treated as positional arguments\n",
    "* `ByRow` and `AsTable` work exactly like discussed for operations on data frames in 05_columns.ipynb\n",
    "* you can automatically groupby again the result of `combine`, `select` etc. by passing `ungroup=false` keyword argument to them\n",
    "* similarly `keepkeys` keyword argument allows you to drop grouping columns from the resulting data frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also allowed to pass a function to all these functions (also - as a special case, as a first argument). In this case the return value can be a table. In particular it allows for an easy dropping of groups if you return an empty table from the function.\n",
    "\n",
    "If you pass a function you can use a `do` block syntax. In case of passing a function it gets a `SubDataFrame` as its argument.\n",
    "\n",
    "Here is an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>n</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><p>1 rows × 2 columns</p><tr><th>1</th><td>c</td><td>31</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & n\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & c & 31 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id   \u001b[0m\u001b[1m n     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼─────────────\n",
       "   1 │ c        31"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine(groupby(x, :id)) do sdf\n",
    "    n = nrow(sdf)\n",
    "    n < 25 ? DataFrame() : DataFrame(n=n) # drop groups with low number of rows\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also produce multiple columns in a single operation, e.g.:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>val</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>1</td><td>2</td></tr><tr><th>3</th><td>2</td><td>3</td></tr><tr><th>4</th><td>2</td><td>4</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & val\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 1 & 2 \\\\\n",
       "\t3 & 2 & 3 \\\\\n",
       "\t4 & 2 & 4 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m val   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     1      2\n",
       "   3 │     2      3\n",
       "   4 │     2      4"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = DataFrame(id=[1,1,2,2], val=[1,2,3,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x1</th><th>x2</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>2 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>2</td></tr><tr><th>2</th><td>2</td><td>3</td><td>4</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & x1 & x2\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 2 \\\\\n",
       "\t2 & 2 & 3 & 4 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x1    \u001b[0m\u001b[1m x2    \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼─────────────────────\n",
       "   1 │     1      1      2\n",
       "   2 │     2      3      4"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@pipe df |> groupby(_, :id) |> combine(_, :val => (x -> [x]) => AsTable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>c1</th><th>c2</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>2 rows × 3 columns</p><tr><th>1</th><td>1</td><td>1</td><td>2</td></tr><tr><th>2</th><td>2</td><td>3</td><td>4</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& id & c1 & c2\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 2 \\\\\n",
       "\t2 & 2 & 3 & 4 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m c1    \u001b[0m\u001b[1m c2    \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼─────────────────────\n",
       "   1 │     1      1      2\n",
       "   2 │     2      3      4"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@pipe df |> groupby(_, :id) |> combine(_, :val => (x -> [x]) => [:c1, :c2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aggregation of a data frame using `mapcols`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>10 rows × 10 columns (omitted printing of 2 columns)</p><tr><th>1</th><td>0.0966498</td><td>0.759763</td><td>0.562443</td><td>0.38894</td><td>0.411924</td><td>0.538266</td><td>0.297264</td><td>0.711936</td></tr><tr><th>2</th><td>0.504155</td><td>0.111741</td><td>0.222208</td><td>0.512403</td><td>0.364941</td><td>0.806049</td><td>0.755719</td><td>0.630507</td></tr><tr><th>3</th><td>0.131687</td><td>0.349729</td><td>0.384267</td><td>0.0517748</td><td>0.124143</td><td>0.543742</td><td>0.197219</td><td>0.807013</td></tr><tr><th>4</th><td>0.624309</td><td>0.839581</td><td>0.555312</td><td>0.591734</td><td>0.178142</td><td>0.785604</td><td>0.0835652</td><td>0.570699</td></tr><tr><th>5</th><td>0.0522465</td><td>0.899302</td><td>0.534734</td><td>0.487683</td><td>0.443829</td><td>0.436251</td><td>0.388639</td><td>0.508346</td></tr><tr><th>6</th><td>0.847171</td><td>0.613108</td><td>0.883328</td><td>0.125407</td><td>0.433376</td><td>0.642765</td><td>0.710859</td><td>0.247029</td></tr><tr><th>7</th><td>0.323028</td><td>0.0143271</td><td>0.70193</td><td>0.908305</td><td>0.975363</td><td>0.863064</td><td>0.625343</td><td>0.668361</td></tr><tr><th>8</th><td>0.477463</td><td>0.254007</td><td>0.674406</td><td>0.63871</td><td>0.891207</td><td>0.307283</td><td>0.430385</td><td>0.843707</td></tr><tr><th>9</th><td>0.575513</td><td>0.312003</td><td>0.00964212</td><td>0.598696</td><td>0.790431</td><td>0.75908</td><td>0.939931</td><td>0.666364</td></tr><tr><th>10</th><td>0.0583058</td><td>0.628603</td><td>0.658883</td><td>0.974989</td><td>0.257514</td><td>0.753738</td><td>0.880556</td><td>0.716262</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.0966498 & 0.759763 & 0.562443 & 0.38894 & 0.411924 & 0.538266 & 0.297264 & 0.711936 & $\\dots$ \\\\\n",
       "\t2 & 0.504155 & 0.111741 & 0.222208 & 0.512403 & 0.364941 & 0.806049 & 0.755719 & 0.630507 & $\\dots$ \\\\\n",
       "\t3 & 0.131687 & 0.349729 & 0.384267 & 0.0517748 & 0.124143 & 0.543742 & 0.197219 & 0.807013 & $\\dots$ \\\\\n",
       "\t4 & 0.624309 & 0.839581 & 0.555312 & 0.591734 & 0.178142 & 0.785604 & 0.0835652 & 0.570699 & $\\dots$ \\\\\n",
       "\t5 & 0.0522465 & 0.899302 & 0.534734 & 0.487683 & 0.443829 & 0.436251 & 0.388639 & 0.508346 & $\\dots$ \\\\\n",
       "\t6 & 0.847171 & 0.613108 & 0.883328 & 0.125407 & 0.433376 & 0.642765 & 0.710859 & 0.247029 & $\\dots$ \\\\\n",
       "\t7 & 0.323028 & 0.0143271 & 0.70193 & 0.908305 & 0.975363 & 0.863064 & 0.625343 & 0.668361 & $\\dots$ \\\\\n",
       "\t8 & 0.477463 & 0.254007 & 0.674406 & 0.63871 & 0.891207 & 0.307283 & 0.430385 & 0.843707 & $\\dots$ \\\\\n",
       "\t9 & 0.575513 & 0.312003 & 0.00964212 & 0.598696 & 0.790431 & 0.75908 & 0.939931 & 0.666364 & $\\dots$ \\\\\n",
       "\t10 & 0.0583058 & 0.628603 & 0.658883 & 0.974989 & 0.257514 & 0.753738 & 0.880556 & 0.716262 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m10×10 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1        \u001b[0m\u001b[1m x2        \u001b[0m\u001b[1m x3         \u001b[0m\u001b[1m x4        \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7    \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64    \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float6\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.0966498  0.759763   0.562443    0.38894    0.411924  0.538266  0.2972 ⋯\n",
       "   2 │ 0.504155   0.111741   0.222208    0.512403   0.364941  0.806049  0.7557\n",
       "   3 │ 0.131687   0.349729   0.384267    0.0517748  0.124143  0.543742  0.1972\n",
       "  ⋮  │     ⋮          ⋮          ⋮           ⋮         ⋮         ⋮          ⋮  ⋱\n",
       "   8 │ 0.477463   0.254007   0.674406    0.63871    0.891207  0.307283  0.4303\n",
       "   9 │ 0.575513   0.312003   0.00964212  0.598696   0.790431  0.75908   0.9399 ⋯\n",
       "  10 │ 0.0583058  0.628603   0.658883    0.974989   0.257514  0.753738  0.8805\n",
       "\u001b[31m                                                    4 columns and 4 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(rand(10, 10), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>1 rows × 10 columns (omitted printing of 2 columns)</p><tr><th>1</th><td>0.369053</td><td>0.478217</td><td>0.518715</td><td>0.527864</td><td>0.487087</td><td>0.643584</td><td>0.530948</td><td>0.637022</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.369053 & 0.478217 & 0.518715 & 0.527864 & 0.487087 & 0.643584 & 0.530948 & 0.637022 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×10 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.369053  0.478217  0.518715  0.527864  0.487087  0.643584  0.530948  0 ⋯\n",
       "\u001b[31m                                                               3 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapcols(mean, x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mapping rows and columns using `eachcol` and `eachrow`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element Array{Float64,1}:\n",
       " 0.3690529289039059\n",
       " 0.47821651924120656\n",
       " 0.5187154604905209\n",
       " 0.5278640303308\n",
       " 0.4870868703188103\n",
       " 0.6435841691148785\n",
       " 0.530948128024424\n",
       " 0.6370223121108235\n",
       " 0.4188901036310245\n",
       " 0.4796283364142309"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "map(mean, eachcol(x)) # map a function over each column and return a vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x1: 0.3690529289039059\n",
      "x2: 0.47821651924120656\n",
      "x3: 0.5187154604905209\n",
      "x4: 0.5278640303308\n",
      "x5: 0.4870868703188103\n",
      "x6: 0.6435841691148785\n",
      "x7: 0.530948128024424\n",
      "x8: 0.6370223121108235\n",
      "x9: 0.4188901036310245\n",
      "x10: 0.4796283364142309\n"
     ]
    }
   ],
   "source": [
    "# an iteration returns a Pair with column name and values\n",
    "foreach(c -> println(c[1], \": \", mean(c[2])), pairs(eachcol(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element Array{Float64,1}:\n",
       "  0.1272105040056249\n",
       "  4.511825317637913\n",
       "  0.3765416376018312\n",
       "  0.743596019284321\n",
       "  0.0580967010964789\n",
       "  1.3817632340717454\n",
       " 22.546594714959323\n",
       "  1.8797217080845827\n",
       "  1.844574399600223\n",
       "  0.0927546582403427"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now the returned value is DataFrameRow which works as a NamedTuple but is a view to a parent DataFrame\n",
    "map(r -> r.x1/r.x2, eachrow(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p>10×10 DataFrameRows</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>0.0966498</td><td>0.759763</td><td>0.562443</td><td>0.38894</td><td>0.411924</td><td>0.538266</td><td>0.297264</td><td>0.711936</td></tr><tr><th>2</th><td>0.504155</td><td>0.111741</td><td>0.222208</td><td>0.512403</td><td>0.364941</td><td>0.806049</td><td>0.755719</td><td>0.630507</td></tr><tr><th>3</th><td>0.131687</td><td>0.349729</td><td>0.384267</td><td>0.0517748</td><td>0.124143</td><td>0.543742</td><td>0.197219</td><td>0.807013</td></tr><tr><th>4</th><td>0.624309</td><td>0.839581</td><td>0.555312</td><td>0.591734</td><td>0.178142</td><td>0.785604</td><td>0.0835652</td><td>0.570699</td></tr><tr><th>5</th><td>0.0522465</td><td>0.899302</td><td>0.534734</td><td>0.487683</td><td>0.443829</td><td>0.436251</td><td>0.388639</td><td>0.508346</td></tr><tr><th>6</th><td>0.847171</td><td>0.613108</td><td>0.883328</td><td>0.125407</td><td>0.433376</td><td>0.642765</td><td>0.710859</td><td>0.247029</td></tr><tr><th>7</th><td>0.323028</td><td>0.0143271</td><td>0.70193</td><td>0.908305</td><td>0.975363</td><td>0.863064</td><td>0.625343</td><td>0.668361</td></tr><tr><th>8</th><td>0.477463</td><td>0.254007</td><td>0.674406</td><td>0.63871</td><td>0.891207</td><td>0.307283</td><td>0.430385</td><td>0.843707</td></tr><tr><th>9</th><td>0.575513</td><td>0.312003</td><td>0.00964212</td><td>0.598696</td><td>0.790431</td><td>0.75908</td><td>0.939931</td><td>0.666364</td></tr><tr><th>10</th><td>0.0583058</td><td>0.628603</td><td>0.658883</td><td>0.974989</td><td>0.257514</td><td>0.753738</td><td>0.880556</td><td>0.716262</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.0966498 & 0.759763 & 0.562443 & 0.38894 & 0.411924 & 0.538266 & 0.297264 & 0.711936 & $\\dots$ \\\\\n",
       "\t2 & 0.504155 & 0.111741 & 0.222208 & 0.512403 & 0.364941 & 0.806049 & 0.755719 & 0.630507 & $\\dots$ \\\\\n",
       "\t3 & 0.131687 & 0.349729 & 0.384267 & 0.0517748 & 0.124143 & 0.543742 & 0.197219 & 0.807013 & $\\dots$ \\\\\n",
       "\t4 & 0.624309 & 0.839581 & 0.555312 & 0.591734 & 0.178142 & 0.785604 & 0.0835652 & 0.570699 & $\\dots$ \\\\\n",
       "\t5 & 0.0522465 & 0.899302 & 0.534734 & 0.487683 & 0.443829 & 0.436251 & 0.388639 & 0.508346 & $\\dots$ \\\\\n",
       "\t6 & 0.847171 & 0.613108 & 0.883328 & 0.125407 & 0.433376 & 0.642765 & 0.710859 & 0.247029 & $\\dots$ \\\\\n",
       "\t7 & 0.323028 & 0.0143271 & 0.70193 & 0.908305 & 0.975363 & 0.863064 & 0.625343 & 0.668361 & $\\dots$ \\\\\n",
       "\t8 & 0.477463 & 0.254007 & 0.674406 & 0.63871 & 0.891207 & 0.307283 & 0.430385 & 0.843707 & $\\dots$ \\\\\n",
       "\t9 & 0.575513 & 0.312003 & 0.00964212 & 0.598696 & 0.790431 & 0.75908 & 0.939931 & 0.666364 & $\\dots$ \\\\\n",
       "\t10 & 0.0583058 & 0.628603 & 0.658883 & 0.974989 & 0.257514 & 0.753738 & 0.880556 & 0.716262 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m10×10 DataFrameRows\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1        \u001b[0m\u001b[1m x2        \u001b[0m\u001b[1m x3         \u001b[0m\u001b[1m x4        \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7    \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64    \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float6\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.0966498  0.759763   0.562443    0.38894    0.411924  0.538266  0.2972 ⋯\n",
       "   2 │ 0.504155   0.111741   0.222208    0.512403   0.364941  0.806049  0.7557\n",
       "   3 │ 0.131687   0.349729   0.384267    0.0517748  0.124143  0.543742  0.1972\n",
       "  ⋮  │     ⋮          ⋮          ⋮           ⋮         ⋮         ⋮          ⋮  ⋱\n",
       "   8 │ 0.477463   0.254007   0.674406    0.63871    0.891207  0.307283  0.4303\n",
       "   9 │ 0.575513   0.312003   0.00964212  0.598696   0.790431  0.75908   0.9399 ⋯\n",
       "  10 │ 0.0583058  0.628603   0.658883    0.974989   0.257514  0.753738  0.8805\n",
       "\u001b[31m                                                    4 columns and 4 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# it prints like a data frame, only the caption is different so that you know the type of the object\n",
    "er = eachrow(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element Array{Float64,1}:\n",
       " 0.09664982449109094\n",
       " 0.5041549474701863\n",
       " 0.13168736619075805\n",
       " 0.624309410304785\n",
       " 0.05224650361175809\n",
       " 0.8471705529645883\n",
       " 0.3230280600962392\n",
       " 0.4774632976198079\n",
       " 0.575513492763198\n",
       " 0.058305833526646644"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "er.x1 # you can access columns of a parent data frame directly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p>10×10 DataFrameColumns</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>0.0966498</td><td>0.759763</td><td>0.562443</td><td>0.38894</td><td>0.411924</td><td>0.538266</td><td>0.297264</td><td>0.711936</td></tr><tr><th>2</th><td>0.504155</td><td>0.111741</td><td>0.222208</td><td>0.512403</td><td>0.364941</td><td>0.806049</td><td>0.755719</td><td>0.630507</td></tr><tr><th>3</th><td>0.131687</td><td>0.349729</td><td>0.384267</td><td>0.0517748</td><td>0.124143</td><td>0.543742</td><td>0.197219</td><td>0.807013</td></tr><tr><th>4</th><td>0.624309</td><td>0.839581</td><td>0.555312</td><td>0.591734</td><td>0.178142</td><td>0.785604</td><td>0.0835652</td><td>0.570699</td></tr><tr><th>5</th><td>0.0522465</td><td>0.899302</td><td>0.534734</td><td>0.487683</td><td>0.443829</td><td>0.436251</td><td>0.388639</td><td>0.508346</td></tr><tr><th>6</th><td>0.847171</td><td>0.613108</td><td>0.883328</td><td>0.125407</td><td>0.433376</td><td>0.642765</td><td>0.710859</td><td>0.247029</td></tr><tr><th>7</th><td>0.323028</td><td>0.0143271</td><td>0.70193</td><td>0.908305</td><td>0.975363</td><td>0.863064</td><td>0.625343</td><td>0.668361</td></tr><tr><th>8</th><td>0.477463</td><td>0.254007</td><td>0.674406</td><td>0.63871</td><td>0.891207</td><td>0.307283</td><td>0.430385</td><td>0.843707</td></tr><tr><th>9</th><td>0.575513</td><td>0.312003</td><td>0.00964212</td><td>0.598696</td><td>0.790431</td><td>0.75908</td><td>0.939931</td><td>0.666364</td></tr><tr><th>10</th><td>0.0583058</td><td>0.628603</td><td>0.658883</td><td>0.974989</td><td>0.257514</td><td>0.753738</td><td>0.880556</td><td>0.716262</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.0966498 & 0.759763 & 0.562443 & 0.38894 & 0.411924 & 0.538266 & 0.297264 & 0.711936 & $\\dots$ \\\\\n",
       "\t2 & 0.504155 & 0.111741 & 0.222208 & 0.512403 & 0.364941 & 0.806049 & 0.755719 & 0.630507 & $\\dots$ \\\\\n",
       "\t3 & 0.131687 & 0.349729 & 0.384267 & 0.0517748 & 0.124143 & 0.543742 & 0.197219 & 0.807013 & $\\dots$ \\\\\n",
       "\t4 & 0.624309 & 0.839581 & 0.555312 & 0.591734 & 0.178142 & 0.785604 & 0.0835652 & 0.570699 & $\\dots$ \\\\\n",
       "\t5 & 0.0522465 & 0.899302 & 0.534734 & 0.487683 & 0.443829 & 0.436251 & 0.388639 & 0.508346 & $\\dots$ \\\\\n",
       "\t6 & 0.847171 & 0.613108 & 0.883328 & 0.125407 & 0.433376 & 0.642765 & 0.710859 & 0.247029 & $\\dots$ \\\\\n",
       "\t7 & 0.323028 & 0.0143271 & 0.70193 & 0.908305 & 0.975363 & 0.863064 & 0.625343 & 0.668361 & $\\dots$ \\\\\n",
       "\t8 & 0.477463 & 0.254007 & 0.674406 & 0.63871 & 0.891207 & 0.307283 & 0.430385 & 0.843707 & $\\dots$ \\\\\n",
       "\t9 & 0.575513 & 0.312003 & 0.00964212 & 0.598696 & 0.790431 & 0.75908 & 0.939931 & 0.666364 & $\\dots$ \\\\\n",
       "\t10 & 0.0583058 & 0.628603 & 0.658883 & 0.974989 & 0.257514 & 0.753738 & 0.880556 & 0.716262 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m10×10 DataFrameColumns\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1        \u001b[0m\u001b[1m x2        \u001b[0m\u001b[1m x3         \u001b[0m\u001b[1m x4        \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7    \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64    \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float6\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.0966498  0.759763   0.562443    0.38894    0.411924  0.538266  0.2972 ⋯\n",
       "   2 │ 0.504155   0.111741   0.222208    0.512403   0.364941  0.806049  0.7557\n",
       "   3 │ 0.131687   0.349729   0.384267    0.0517748  0.124143  0.543742  0.1972\n",
       "  ⋮  │     ⋮          ⋮          ⋮           ⋮         ⋮         ⋮          ⋮  ⋱\n",
       "   8 │ 0.477463   0.254007   0.674406    0.63871    0.891207  0.307283  0.4303\n",
       "   9 │ 0.575513   0.312003   0.00964212  0.598696   0.790431  0.75908   0.9399 ⋯\n",
       "  10 │ 0.0583058  0.628603   0.658883    0.974989   0.257514  0.753738  0.8805\n",
       "\u001b[31m                                                    4 columns and 4 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# it prints like a data frame, only the caption is different so that you know the type of the object\n",
    "ec = eachcol(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element Array{Float64,1}:\n",
       " 0.09664982449109094\n",
       " 0.5041549474701863\n",
       " 0.13168736619075805\n",
       " 0.624309410304785\n",
       " 0.05224650361175809\n",
       " 0.8471705529645883\n",
       " 0.3230280600962392\n",
       " 0.4774632976198079\n",
       " 0.575513492763198\n",
       " 0.058305833526646644"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ec.x1 # you can access columns of a parent data frame directly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transposing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can transpose a data frame using `permutedims`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>3 rows × 4 columns</p><tr><th>1</th><td>1</td><td>4</td><td>7</td><td>10</td></tr><tr><th>2</th><td>2</td><td>5</td><td>8</td><td>11</td></tr><tr><th>3</th><td>3</td><td>6</td><td>9</td><td>12</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccc}\n",
       "\t& x1 & x2 & x3 & x4\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 4 & 7 & 10 \\\\\n",
       "\t2 & 2 & 5 & 8 & 11 \\\\\n",
       "\t3 & 3 & 6 & 9 & 12 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×4 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1    \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3    \u001b[0m\u001b[1m x4    \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼────────────────────────────\n",
       "   1 │     1      4      7     10\n",
       "   2 │     2      5      8     11\n",
       "   3 │     3      6      9     12"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = DataFrame(reshape(1:12, 3, 4), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3-element Array{String,1}:\n",
       " \"a\"\n",
       " \"b\"\n",
       " \"c\""
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.names = [\"a\", \"b\", \"c\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>names</th><th>a</th><th>b</th><th>c</th></tr><tr><th></th><th>String</th><th>Int64</th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>4 rows × 4 columns</p><tr><th>1</th><td>x1</td><td>1</td><td>2</td><td>3</td></tr><tr><th>2</th><td>x2</td><td>4</td><td>5</td><td>6</td></tr><tr><th>3</th><td>x3</td><td>7</td><td>8</td><td>9</td></tr><tr><th>4</th><td>x4</td><td>10</td><td>11</td><td>12</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccc}\n",
       "\t& names & a & b & c\\\\\n",
       "\t\\hline\n",
       "\t& String & Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & x1 & 1 & 2 & 3 \\\\\n",
       "\t2 & x2 & 4 & 5 & 6 \\\\\n",
       "\t3 & x3 & 7 & 8 & 9 \\\\\n",
       "\t4 & x4 & 10 & 11 & 12 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×4 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m names  \u001b[0m\u001b[1m a     \u001b[0m\u001b[1m b     \u001b[0m\u001b[1m c     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m String \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼─────────────────────────────\n",
       "   1 │ x1          1      2      3\n",
       "   2 │ x2          4      5      6\n",
       "   3 │ x3          7      8      9\n",
       "   4 │ x4         10     11     12"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permutedims(df, :names)"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.5.3",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
